#!/usr/bin/env python

# (c) 2007-2011 Helsinki University of Technology and University of Helsinki
# Licensed under the MIT license; see LICENSE.txt for more information.

from harava.config import Config, DEFAULT_CONFNAME, DEFAULT_CONFDIR
from harava.lookup import LookupService
from harava.model import ModelService
from harava.session import Session
from harava.memory import memory, resident

import resource
import time
import sys
import optparse

#import gc
#gc.set_debug(gc.DEBUG_LEAK)
#gc.set_debug(gc.DEBUG_UNCOLLECTABLE)

def main():
  parser = optparse.OptionParser()
  parser.add_option('-c', '--cached', action='store_true', dest='cached', help="Use cached copies of web documents when available (use after running with --keep)")
  parser.add_option('-C', '--confname', type='string', dest='confname', default=DEFAULT_CONFNAME, help="Configuration name (default: %s)" % DEFAULT_CONFNAME)
  parser.add_option('-D', '--confdir', type='string', dest='confdir', default=DEFAULT_CONFDIR, help="Configuration directory (default: %s)" % DEFAULT_CONFDIR)
  parser.add_option('-l', '--limit', type='int', dest='limit', help='Process at most this many documents from each source')
  parser.add_option('-k', '--keep', action='store_true', dest='keep', help="Keep fetched documents so they can be used with --cached")
  parser.add_option('-n', '--notes', action='store_true', dest='notes', help="Include NOTE level messages in reports")
  parser.add_option('-p', '--progress', action='store_true', dest='progress', help="Print progress status on stderr every 5 seconds")
  parser.add_option('-r', '--report', type='int', dest='report', default=2, help="Reporting level: 0=none, 1=summary, 2=all (default:2)")
  parser.add_option('-s', '--source', type='string', dest='source', default='', help="Sources to process (separate multiple with commas)")
  parser.add_option('-u', '--url', type='string', dest='url', default='', help="Single URL to process, creating a document report on stdout")
  parser.add_option('-v', '--verbose', action='store_true', dest='verbose', help="Print verbose status and statistics on stderr")
  
  (options, remainingArgs) = parser.parse_args()

  cfg = Config(confdir=options.confdir, confname=options.confname)
  model = ModelService(cfg, options.verbose)
  lookup = LookupService(cfg, model)
  sessions = []
  srcs = [s.strip() for s in options.source.split(',') if s.strip() != '']
  
  if options.verbose:
    print >>sys.stderr, "Configured sources:", ' '.join([s.getName() for s in cfg.getSources()])
    print >>sys.stderr, "Requested sources:", ' '.join(srcs)
  
  # create and startup all sessions
  for source in cfg.getSources():
    if srcs and source.getName() not in srcs:
      continue
  
    sess = Session(cfg, lookup, model, source, limit=options.limit, report=options.report, keep=options.keep, cached=options.cached, progress=options.progress, url=options.url, verbose=options.verbose, notes=options.notes)
    sess.start()
    sessions.append(sess)
  # wait for sessions to finish
  for sess in sessions:
    sess.close()
  
  if options.verbose:
    lookup.printStatistics()
    print >>sys.stderr, "Resources spent: virtual mem %d MB / resident %d MB, CPU time %d sec, wall clock time %d sec" % (
      int(memory() / (1024 * 1024)),
      int(resident() / (1024 * 1024)),
      int(resource.getrusage(resource.RUSAGE_SELF)[0]),
      int(time.time() - starttime)
    )

if __name__ == '__main__':
  starttime = time.time()

  main()

